import os
import pickle
import time
import json
import concurrent.futures
from threading import Lock
import re

import requests
from selenium import webdriver
from selenium.common import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from openpyxl import Workbook, load_workbook

# 创建锁对象用于保护文件操作
excel_lock = Lock()




def get_ip():
    """获取可用的代理IP，直到成功为止"""
    time.sleep(2)

    def test_ip(ip):
        try:
            proxies = {
                'http': f'http://{ip}',
                'https': f'http://{ip}'
            }
            response = requests.get('http://www.baidu.com',
                                    proxies=proxies,
                                    timeout=10)
            return response.status_code == 200
        except:
            return False

    while True:
        try:
            api = '' #此处输入获取代理服务器的API
            ip = requests.get(api).text.strip()

            if test_ip(ip):
                print(f"找到可用代理IP: {ip}")
                return ip
            else:
                print(f"代理IP {ip} 不可用，继续尝试...")
                time.sleep(2)  # 避免频繁请求

        except Exception as e:
            print(f"获取代理IP时出错: {e}")
            time.sleep(2)  # 出错时等待后重试


def save_score_data(file_name, school_id, province, year, subject, batch, score_data):
    """保存数据到Excel（线程安全版本）"""
    with excel_lock:  # 使用锁保护文件操作
        try:
            excel_dir = r'D:\school\score_datas'
            if not os.path.exists(excel_dir):
                os.makedirs(excel_dir)

            excel_path = os.path.join(excel_dir, f'{file_name}_subject_score.xlsx')

            # 如果文件不存在，创建新文件并添加表头
            if not os.path.exists(excel_path):
                wb = Workbook()
                ws = wb.active
                ws.title = "分数线数据"
                headers = ["学校ID", "省份", "年份", "批次", "分数线数据"]
                for col, header in enumerate(headers, 1):
                    ws.cell(row=1, column=col, value=header)
            else:
                wb = load_workbook(excel_path)
                ws = wb.active

            # 获取下一个可用行
            next_row = ws.max_row + 1

            # 添加新数据
            ws.cell(row=next_row, column=1, value=school_id)  # 学校ID
            ws.cell(row=next_row, column=2, value=province)
            ws.cell(row=next_row, column=3, value=year)
            ws.cell(row=next_row, column=4, value=subject)
            ws.cell(row=next_row, column=5, value=batch)
            ws.cell(row=next_row, column=6, value=str(score_data))

            # 保存文件
            wb.save(excel_path)
            print(f"数据已保存到第 {next_row} 行")

        except Exception as e:
            print(f"保存数据时出错: {e}")


def save_failed_data(file_name, school_id, province, year, subject, batch):
    """保存爬取失败的数据记录（线程安全版本）"""
    with excel_lock:  # 使用锁保护文件操作
        try:
            excel_dir = r'D:\school\score_datas'
            if not os.path.exists(excel_dir):
                os.makedirs(excel_dir)

            excel_path = os.path.join(excel_dir, f'{file_name}_subject_failed.xlsx')

            # 如果文件不存在，创建新文件并添加表头
            if not os.path.exists(excel_path):
                wb = Workbook()
                ws = wb.active
                ws.title = "失败记录"
                headers = ["学校ID", "省份", "年份", "科目", "批次", "失败原因"]
                for col, header in enumerate(headers, 1):
                    ws.cell(row=1, column=col, value=header)
            else:
                wb = load_workbook(excel_path)
                ws = wb.active

            # 获取下一个可用行
            next_row = ws.max_row + 1

            # 添加新数据
            ws.cell(row=next_row, column=1, value=school_id)  # 学校ID
            ws.cell(row=next_row, column=2, value=province)
            ws.cell(row=next_row, column=3, value=year)
            ws.cell(row=next_row, column=4, value=subject)
            ws.cell(row=next_row, column=5, value=batch)
            ws.cell(row=next_row, column=6, value="数据获取失败")

            # 保存文件
            wb.save(excel_path)
            print(f"失败记录已保存到第 {next_row} 行")

        except Exception as e:
            print(f"保存失败记录时出错: {e}")


def load_cookies(driver, cookie_file):
    """加载cookies"""
    try:
        with open(cookie_file, 'r', encoding='utf-8') as f:
            cookies = json.loads(f.read())
        for cookie in cookies:
            driver.add_cookie(cookie)
        return True
    except Exception as e:
        print(f"加载cookies失败：{str(e)}")
        return False


def get_subject_types(type_list):
    type_lists = re.findall(r'[\u4e00-\u9fa5]+', type_list)
    return type_lists




def scroll_until_element_found(driver, selector, selector_type=By.CSS_SELECTOR, max_scrolls=2, scroll_pause=0.5):

    # 已滚动的总高度
    scroll_height = 0

    # 记录上一次的滚动高度，用于检测是否已到达页面底部
    last_height = driver.execute_script("return document.body.scrollHeight")

    for i in range(max_scrolls):
        try:
            # 尝试找到元素
            element = driver.find_element(selector_type, selector)
            print(f"找到元素！位置: {element.location}")
            return element
        except NoSuchElementException:
            # 元素未找到，继续滚动
            # 每次滚动一个视口高度
            driver.execute_script("window.scrollBy(0, window.innerHeight);")
            scroll_height += driver.execute_script("return window.innerHeight")
            print(f"滚动中... 当前滚动高度: {scroll_height}px")

            # 暂停，等待内容加载
            time.sleep(scroll_pause)

            # 检查是否已滚动到底部
            new_height = driver.execute_script("return document.body.scrollHeight")
            if new_height == last_height:
                print("已滚动到页面底部，未找到元素")
                break
            last_height = new_height

    print(f"已达到最大滚动次数({max_scrolls})或滚动到底部，未找到元素")
    return None


# 使用示例
# element = scroll_until_element_found(driver, "你要查找的选择器", By.CSS_SELECTOR)
# 将处理单个学校的逻辑封装到函数中，用于多线程执行

def getdata(driver, province, school_id, province_index, year, subject, batch):
    # 向下滚动3个单位(假设每单位是100像素)

    time.sleep(5)
    try:
        score_elements = driver.find_elements(By.XPATH,'//*[@id="scoreline"]/div/div[2]/table/tbody')
        data_found = False
        for score_element in score_elements:
            try:
                score_data = score_element.text.split('\n')
                if score_data and len(score_data) > 0:
                    print(f"分数线数据：{score_data}")
                    data_found = True
                    return score_data
            except Exception as e:
                print(f"处理元素时出错: {e}")

        # 如果第一个XPath没有找到数据，尝试使用第二个XPath
        if not data_found:
            print("使用备选XPath查找分数线数据...")
            score_elements = driver.find_elements(By.XPATH, '//*[@id="scoreline"]/div/div[3]/table/tbody')
            for score_element in score_elements:
                try:
                    score_data = score_element.text.split('\n')
                    if score_data and len(score_data) > 0:
                        print(f"分数线数据：{score_data}")
                        return score_data
                except Exception as e:
                    print(f"处理备选元素时出错: {e}")
    except Exception as e:
        print(f"查找分数线元素时出错: {e}")
        save_failed_data(province, school_id, province_index, year, subject, batch)
    time.sleep(5)

def has_next_page(driver):
    try:

        # 查找下一页按钮并获取其aria-disabled属性
        next_page_button = driver.find_element(By.XPATH,'//li[@title="Next Page" and contains(@class, "ant-pagination-next")]')

        # 获取aria-disabled属性值
        is_disabled = next_page_button.get_attribute('aria-disabled')

        # 如果aria-disabled为"false"则表示有下一页
        return is_disabled.lower() == "false"
    except:
        # 如果找不到元素或发生其他错误，默认认为没有下一页
        print("无法找到下一页按钮或检查其状态")
        return False


def process_school(school):
    school_id = school[0]
    school_name = school[1]
    last_data = None
    try:
        # 初始化浏览器
        service = Service(executable_path='D:\chromedriver_win32 (1)\chromedriver.exe')
        #ip = get_ip() #上面如果输入了代理服务器API则将这行和下面第二行注释取消

        chrome_options = webdriver.ChromeOptions()

        #chrome_options.add_argument('--proxy-server=http://' + str(ip))

        # 创建 Chrome 浏览器实例
        driver = webdriver.Chrome(service=service,options=chrome_options)
        # 访问网页并加载cookies
        url = f'https://www.gaokao.cn/school/{school_id}/provinceline'
        driver.get(url)
        try:
            buttom_xpath='/html/body/div[2]/div/div[2]/div/div[2]/div[3]/button'
            parent=WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.XPATH, "/html/body/div[2]/div/div[2]/div/div[2]")))
            button=WebDriverWait(driver,5).until(EC.element_to_be_clickable((By.XPATH, buttom_xpath)))
            button.click()
        except Exception as e:
            print("wu")

        driver.maximize_window()
        if not load_cookies(driver, 'cookies_juejin.json'):
            return

        driver.refresh()
        time.sleep(10)

        # 遍历省份
        # 获取省份
        province_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[3]
        province_dropdown.click()
        province_opt = WebDriverWait(driver, 5).until(EC.visibility_of_element_located(
            (By.XPATH, '//*[@id="scoreline"]/div/div[1]/div/div[1]/div[2]/div/div/div')))
        province_list = driver.find_elements(By.XPATH, '//*[@id="scoreline"]/div/div[1]/div/div[1]/div[2]/div/div/div')[
            0].text
        provinces = re.findall(r'[\u4e00-\u9fa5]+', province_list)
        driver.execute_script("window.scrollBy(0, 300);")
        initial_position = driver.execute_script("return window.pageYOffset;")
        time.sleep(1)
        province_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[3]
        province_dropdown.click()
        time.sleep(2)
        # 获取垂直滚动位置
        province_index = 1
        for province in provinces:
            try:
                # 选择省份
                driver.execute_script(f"window.scrollTo(0, {initial_position});")
                province_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[3]
                province_dropdown.click()
                province_xpath = f'//*[@id="scoreline"]/div/div[1]/div/div[1]/div[2]/div/div/div/div[{province_index}]'
                province_option = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, province_xpath)))
                province_option.click()
                province_index = province_index + 1
                # 遍历年份
                # 获取年份
                year_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[4]
                year_dropdown.click()
                year_opt = WebDriverWait(driver, 5).until(
                    EC.element_to_be_clickable((By.XPATH, '//*[@id="scoreline"]/div/div[1]/div/div[3]/div[2]/div/div')))
                years_str = \
                    driver.find_elements(By.XPATH, '//*[@id="scoreline"]/div/div[1]/div/div[3]/div[2]/div/div')[
                        0].text

                years = years_str.split('\n')
                time.sleep(1)
                year_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[4]
                year_dropdown.click()
                time.sleep(2)
                print(years)
                for year in years:
                    try:
                        # 选择年份
                        driver.execute_script(f"window.scrollTo(0, {initial_position});")
                        year_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[4]
                        year_dropdown.click()

                        year_xpath = f"//li[contains(@class, 'ant-select-dropdown-menu-item') and text()='{year}']"
                        year_option = WebDriverWait(driver, 5).until(
                            EC.element_to_be_clickable((By.XPATH, year_xpath))
                        )
                        year_option.click()
                        time.sleep(1)

                        # 获取科目类型
                        subject_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[5]
                        subject_dropdown.click()
                        sunject_opt = WebDriverWait(driver, 5).until(EC.element_to_be_clickable(
                            (By.XPATH, '//*[@id="scoreline"]/div/div[1]/div/div[5]/div[2]/div/div')))

                        type_list = \
                            driver.find_elements(By.XPATH,
                                                 '//*[@id="scoreline"]/div/div[1]/div/div[5]/div[2]/div/div')[0].text
                        subjects = get_subject_types(type_list)
                        subject_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[5]
                        subject_dropdown.click()
                        time.sleep(2)
                        print(subjects)
                        # 遍历科目类型

                        for subject in subjects:
                            try:
                                driver.execute_script(f"window.scrollTo(0, {initial_position});")
                                subject_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[5]
                                subject_dropdown.click()
                                subject_option = WebDriverWait(driver, 5).until(
                                    EC.presence_of_element_located(
                                        (By.XPATH, f"//li[contains(text(), '{subject}')]"))
                                )
                                subject_option.click()

                                # 获取批次
                                batch_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[6]
                                batch_dropdown.click()
                                batch_opt = WebDriverWait(driver, 5).until(EC.element_to_be_clickable(
                                    (By.XPATH, '//*[@id="scoreline"]/div/div[1]/div/div[7]/div[2]/div/div')))
                                batch_str = driver.find_elements(By.XPATH,
                                                                 '//*[@id="scoreline"]/div/div[1]/div/div[7]/div[2]/div/div')
                                batch_str = [i.text for i in batch_str]
                                batch_list = batch_str[0].split('\n')
                                batch_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[6]
                                batch_dropdown.click()
                                for batch in batch_list:
                                    driver.execute_script(f"window.scrollTo(0, {initial_position});")
                                    if batch == ' ':
                                        print('退出')
                                        break
                                    print(f"处理：{province_index} - {year} - {subject} - {batch}")
                                    batch_dropdown = \
                                        driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[6]
                                    batch_dropdown.click()
                                    time.sleep(3)
                                    batch_option = WebDriverWait(driver, 10).until(EC.presence_of_element_located((
                                        By.XPATH,
                                        f"//li[contains(@class, 'ant-select-dropdown-menu-item') and text()='{batch}']")))
                                    batch_option.click()


                                    # 获取分数线
                                    while True:

                                        score_data = getdata(driver, province, school_id, province_index, year, subject,
                                                             batch)
                                        if score_data==last_data:
                                            driver.refresh()
                                            driver.execute_script(f"window.scrollTo(0, {initial_position});")
                                            province_dropdown = \
                                            driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[3]
                                            province_dropdown.click()
                                            province_xpath = f'//*[@id="scoreline"]/div/div[1]/div/div[1]/div[2]/div/div/div/div[{province_index-1}]'
                                            province_option = WebDriverWait(driver, 5).until(
                                                EC.element_to_be_clickable((By.XPATH, province_xpath)))
                                            province_option.click()
                                            time.sleep(1)

                                            driver.execute_script(f"window.scrollTo(0, {initial_position});")
                                            year_dropdown = \
                                            driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[4]
                                            year_dropdown.click()

                                            year_xpath = f"//li[contains(@class, 'ant-select-dropdown-menu-item') and text()='{year}']"
                                            year_option = WebDriverWait(driver, 5).until(
                                                EC.element_to_be_clickable((By.XPATH, year_xpath))
                                            )
                                            year_option.click()
                                            time.sleep(1)

                                            driver.execute_script(f"window.scrollTo(0, {initial_position});")
                                            subject_dropdown = \
                                            driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[5]
                                            subject_dropdown.click()
                                            subject_option = WebDriverWait(driver, 5).until(
                                                EC.presence_of_element_located(
                                                    (By.XPATH, f"//li[contains(text(), '{subject}')]"))
                                            )
                                            subject_option.click()
                                            time.sleep(1)

                                            batch_dropdown = \
                                                driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[6]
                                            batch_dropdown.click()
                                            time.sleep(3)
                                            batch_option = WebDriverWait(driver, 10).until(
                                                EC.presence_of_element_located((
                                                    By.XPATH,
                                                    f"//li[contains(@class, 'ant-select-dropdown-menu-item') and text()='{batch}']")))
                                            batch_option.click()
                                            score_data=score_data = getdata(driver, province, school_id, province_index, year, subject,
                                                             batch)


                                        save_score_data(school_name, school_id, province, year, subject, batch,
                                                        score_data)
                                        last_data=score_data
                                        if not has_next_page(driver):
                                            print("已到达最后一页，结束遍历")

                                            break
                                        try:
                                            WebDriverWait(driver,5).until(EC.visibility_of_element_located((By.XPATH,'//li[@title="Next Page" and contains(@class, "ant-pagination-next")]')))
                                            next_button = driver.find_element(By.XPATH,
                                                                              '//li[@title="Next Page" and contains(@class, "ant-pagination-next")]')
                                            next_button.click()

                                            # 等待新页面加载
                                            time.sleep(3)  # 简单等待
                                            # 或使用显式等待
                                            # WebDriverWait(driver, 10).until(某个元素可见或其他条件)
                                        except Exception as e:
                                            print(f"点击下一页时出错: {e}")
                                            break



                            except Exception as e:
                                print(f"处理科目 {subject} 时出错：{str(e)}")
                                save_failed_data(school_name, school_id, province, year, subject, '')
                                continue

                    except Exception as e:
                        print(f"处理年份 {year} 时出错：{str(e)}")
                        save_failed_data(school_name, school_id, province, year, '', '')
                        continue

            except Exception as e:
                print(f"处理省份 {province} 时出错：{str(e)}")
                save_failed_data(school_name, school_id, province, '', '', '')
                continue
        driver.quit()
    except Exception as e:
        print(f"程序执行出错：{str(e)}")


def main():
    try:
        with open("school.pkl",'rb') as f:
            schools=pickle.load(f)
        # 使用线程池执行多线程爬取，此处如果没有设置代理服务器则建议并行个数在5以内，防止封IP
        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
            # 提交所有学校任务到线程池
            futures = [executor.submit(process_school, school) for school in schools]

            # 等待所有任务完成
            for future in concurrent.futures.as_completed(futures):
                try:
                    future.result()  # 获取任务结果或异常
                    school_count = school_count + 1
                    if school_count >= 20:
                        break
                except Exception as e:
                    print(f"线程执行错误: {str(e)}")


    finally:
        print("程序执行完毕")



if __name__ == "__main__":
    main()
